package paper.batch;

import paper.http.ClassReLoader;

import java.sql.*;
import java.io.*;
import java.util.Enumeration;
import java.util.Vector;
import java.util.Iterator;
import java.util.zip.ZipFile;
import java.util.zip.GZIPInputStream;
import java.util.zip.ZipEntry;
import java.net.URL;
import java.net.URLConnection;

import com.ice.tar.*;

import paper.db.DB;
import paper.db.PDFFile;
import paper.db.Paper;
import paper.pdf.ExtractTitle;
import paper.pdf.Extractor;
import paper.xml.LinkChaser;
import paper.image.ImageHold;
import paper.batch.MD5;

public class Batch extends Thread {
    static Batch processor;

    public Batch() {
        this.start();
        processor = this;
        System.out.println("Fired off a thread....");
    }

    public synchronized void addFile(String userID, String original, boolean copyrighted, File x, String md5) {
        try {
            Statement st = DB.connection.createStatement();
            st.executeUpdate("INSERT INTO filequeue " +
                             "(type,userid,originalfile,filename,filesize,md5,copyrighted) " +
                             "VALUES ('file','"+ userID +"'," +
                             "'"+ DB.q(Paper.sanitise(original),DB.originalFileMax) +"'," + 
                             "'"+ x.getAbsolutePath() +"'," + 
                             ""+ x.length() +"," + 
                             "'"+ md5 +"'," + 
                             "'"+ (copyrighted?"1":"0")+"')");
            System.out.println("GOT " + md5);
            notify();
        } catch (SQLException e) {
            System.err.println("Failed to add file to queue " + x + " " + e);
        }
    }

    public synchronized void addURL(String userID, String original, boolean copyrighted, String url, int hopCount) {
        try {
            String qUrl = DB.q(url,DB.urlMax);
            Statement st = DB.connection.createStatement();
            ResultSet rs = st.executeQuery("SELECT status FROM urls " +
                                           "WHERE url='"+qUrl+"'");
            if (false && rs.next()) {
                if (hopCount == 0) {
                    System.out.println("Skip " + url + " will be behind horizon");
                    return;
                }
                if (!rs.getString(1).equals(DB.urlStatusHorizon)) {
                    System.out.println("Skip " + url + " done already");
                    return;
                }
                st.executeUpdate("UPDATE urls SET status='"+
                                 DB.urlStatusQueued+"' WHERE " +
                                 "url='"+qUrl+"'");
                System.out.println("Retrying " + url);
            } else {
                st.executeUpdate("INSERT INTO urls (url,status) VALUES " +
                                 "('"+qUrl+"','"+DB.urlStatusQueued+"') ");
                System.out.println("Trying " + url);
            }
            st.executeUpdate("INSERT INTO filequeue " +
                             "(type,userid,originalfile,filename,url,hopcount,copyrighted) " +
                             "VALUES ('url','"+ userID +"'," +
                             "'"+ DB.q(Paper.sanitise(original),DB.originalFileMax) +"'," + 
                             "'','"+ qUrl +"'," + hopCount + "," + 
                             "'"+ (copyrighted?"1":"0")+"')");
            notify();
        } catch (SQLException e) {
            System.err.println("Failed to add url to queue " + url + " " + e);
        }
    }

    public synchronized void doneURL(String url, boolean horizon) {
        try {
            String qUrl = DB.q(url,DB.urlMax);
            Statement st = DB.connection.createStatement();
            st.executeUpdate("UPDATE urls SET status='"+
                             (horizon? DB.urlStatusHorizon :DB.urlStatusDone) +
                             "' WHERE url='"+qUrl+"'");
        } catch (SQLException e) {
            System.err.println("Failed to add update url " + url + " " + e);
        }
    }

    public synchronized void addPaperKey(String key) {
        try {
            Statement st = DB.connection.createStatement();
            ResultSet rs;
            rs = st.executeQuery("SELECT paperkey FROM filequeue WHERE " +
                                 " paperkey='"+ DB.q(key,DB.keyMax) +"'");
            if (!rs.next()) {
                st.executeUpdate("INSERT INTO filequeue (type,paperkey) " +
                                 "VALUES ('key','"+ DB.q(key,DB.keyMax) +"')");
                notify();
            }
        } catch (SQLException e) {
            System.err.println("Failed to add key to queue " + key + " " + e);
        }
    }

    public synchronized void addReprocessAll() {
        try {
            Statement st = DB.connection.createStatement();
            st.executeUpdate("INSERT INTO filequeue (type,md5) " +
                             "SELECT DISTINCT 'md5' AS type, md5 AS filename" +
                             " FROM userpapers WHERE md5<>''");
            notify();
            st.close();
        } catch (SQLException e) {
            System.err.println("Failed to add file to reprocess " + e);
        }
    }

    public synchronized void addReprocessMd5(String md5) {
        try {
            Statement st = DB.connection.createStatement();
            st.executeUpdate("INSERT INTO filequeue (type,md5) " +
                             "VALUES ('md5','"+ DB.q(md5,DB.md5Max) +"')"); 
            notify();
            st.close();
        } catch (SQLException e) {
            System.err.println("Failed to add file to queue " + md5 + " " + e);
        }
    }

    String fileName, md5;
    String originalFile, userID;
    boolean copyrighted;
    int sequence = -1;
    String url, key,type;
    int hopCount;

    public synchronized void getFile() {
        Statement st = null;
        ResultSet rs = null;
        try {
            st = DB.connection.createStatement();
            if (sequence != -1) {
                st.executeUpdate("DELETE FROM filequeue WHERE sequence='"+ sequence +"'");
                sequence++;
            }
            do {
                if (sequence == -1) {
                    try {
                        rs = st.executeQuery("SELECT min(sequence) as ms FROM filequeue");
                        if (!rs.next() || rs.getString("ms") == null) {
//                            System.out.println("Wait....");
                            wait();
//                            System.out.println("Wake-up!");
                        } else {
                            sequence = rs.getInt("ms");
                        }
                    } catch (InterruptedException e) {
                    }
                }
                if (sequence != -1) {
                    rs = st.executeQuery("SELECT * FROM filequeue WHERE sequence='"+ sequence +"'");
                    if (rs.next()) {
                        fileName     = rs.getString(DB.fileName);
                        md5          = rs.getString(DB.md5);
                        userID       = rs.getString(DB.userID);
                        url          = rs.getString(DB.url);
                        key          = rs.getString(DB.paperKey);
                        type         = rs.getString(DB.type);
                        hopCount     = rs.getInt(DB.hopCount);
                        originalFile = rs.getString(DB.originalFile);
                        String     x = rs.getString(DB.copyrighted);
                        copyrighted  = x != null && x.equals("1");
                    } else {
                        sequence = -1; // woops, derailed. try again and wait if necessary.
                    }
                }
            } while(sequence == -1);
        } catch (SQLException e) {
            System.err.println("Failed to take from queue " + sequence + ": " + e);
        } finally {
            try {
                if (rs != null) rs.close();
                if (st != null) st.close();
            } catch(Exception e) {
                System.err.println("Fuck..." + e);
            }
        }
        
    }


    public static void processPublicStream(String original, InputStream is)
        throws IOException {
        processor.processPlain("", original, false, is, null);
    }

    public long processPlain(String userID, String original,
                             boolean copyrighted, InputStream is)
        throws IOException {
        return processPlain(userID, original, copyrighted, is, null);
    }

    public long processPlain(String userID, String original,
                             boolean copyrighted, InputStream is,
                             String originalMd5)
        throws IOException {
        MD5 md = new MD5();
        byte[] buff = new byte[16384];
        BufferedInputStream bis = new BufferedInputStream(is);
        File out = createTempFile();
        BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(out));
        int i;
        long bytes = 0;
        do {
            i = bis.read(buff);
            if (i!= -1) {
                md.update(buff,i);
                bos.write(buff,0,i);
                bytes += i;
            }
        } while (i != -1);
        bos.close();
        bis.close();
        String newMd5 = md.digest();
        addFile(userID, original, copyrighted, out, originalMd5 == null ? newMd5 : originalMd5);
        return bytes;
    }

    static public File createTempFile() throws IOException {
        return File.createTempFile("pBf","huh",new File("nobackup/Papers/tmp"));
    }

    private void processZip(File file) throws IOException {
        ZipFile zf = new ZipFile(file);
        for (Enumeration<ZipEntry> i = (Enumeration<ZipEntry>) zf.entries(); i.hasMoreElements();) {
            ZipEntry ze = i.nextElement();
            InputStream is = zf.getInputStream(ze);
            processPlain(userID, originalFile + " - " + ze.getName(), copyrighted, is);
            is.close();
        }
    }

    private void processTar(File file) throws IOException {
        TarInputStream tin = new TarInputStream(new FileInputStream(file));
        TarEntry tarEntry = tin.getNextEntry();
        while (tarEntry != null) {
            String comm = tarEntry.getName();
            System.out.println("Processing " + comm);
            if (!tarEntry.isDirectory()) {
                processPlain(userID, originalFile + " - " + comm, copyrighted, tin);
            }
            tarEntry = tin.getNextEntry();
        }
        tin.close();
    }

    private void processPostscript(File file) throws IOException {
        String[] ps2pdf = {"/usr/bin/ps2pdf", file.getAbsolutePath(), "-"};
        Process mailer = Runtime.getRuntime().exec(ps2pdf);
        InputStream is = mailer.getInputStream();
        System.out.println("Making pdf from ps");
        processPlain(userID, originalFile + " -> pdf ", copyrighted, is, md5);
        is.close();
    }

    Class extractionClass;

    private void processPDF(File file, String md5) throws IOException {
        Extractor et = null;
        Vector<String> toBeProcessed = new Vector<String>();
//        System.out.println("Processing " + (originalFile!=null ? originalFile : file.toString()));
        try {
            et = (Extractor) extractionClass.getConstructor().newInstance();
        } catch (Exception e) {
            System.err.println("Could not get a new extractor. We're in poo "+ e);
        }
        PDFFile pdf = et.process(toBeProcessed, file,
                                 new PrintWriter(PDFFile.getText(md5)));
        if (pdf == null) {
            return;
        }
        pdf.setMD5(md5);
        if(!pdf.hasTitle){
            pdf.setTitle("MD5: " + md5); // FIXME, need to be set in paper too?
            System.out.println("Generating image...");
            ImageHold.generateImg(PDFFile.getFile(md5));
            ImageHold.makeThumbnail(PDFFile.getImg(md5),PDFFile.getThumbnail(md5));
        }
        pdf.commit();
        for(Iterator<String> i = toBeProcessed.iterator(); i.hasNext();) {
            addPaperKey(i.next());
        }
    }

    private boolean hasBeenProcessed(String md5) throws IOException {
        System.out.println("MD5 = " + md5);
        return PDFFile.loadOnMD5(md5, null) != null;
    }
    
    private void addToUserFiles(String md5) {
        try {
            Statement st = DB.connection.createStatement();
            ResultSet rs = st.executeQuery("SELECT personalnotes FROM userpapers WHERE "+
                                 "userid='"+DB.q(userID,16)+"' and "+
                                 "md5='"+md5+"'");
            if (rs.next()) {
                st.executeUpdate("UPDATE userpapers SET personalnotes='"+
                                 DB.q(rs.getString("personalnotes") +"\n" +
                                      Paper.sanitise(originalFile),65536)+"' WHERE "+
                                 "userid='"+DB.q(userID,16)+"' and "+
                                 "md5='"+md5+"'"); // sequence auto increments
            } else {
                st.executeUpdate("INSERT INTO userpapers (userid,paperkey,md5,personalnotes) VALUES ('" + userID + "','','" + md5 + "','" + DB.q(originalFile,65536) + "')"); // sequence auto increments
            }
            st.close();
        } catch (SQLException e) {
            System.err.println("Failed to add paper  " + e);
        }
    }


    private String getFileType(File unknownFile) throws IOException {
        InputStream is = new FileInputStream(unknownFile);
        byte[] buff = new byte[4];
        if (is.read(buff) != 4) {
            return "empty";
        }
        is.close();
        if (buff[0] == 037 && buff[1] == -117) {
            return "application/x-gzip";
        }
        String x = new String(buff);
        if (x.equals("%!PS")) {
            return "application/postscript";
        }
        if (x.equals("%PDF")) {
            return "application/pdf";
        }
        if (x.equals("PK\003\004")) {      // ?
            return "application/zip";
        }
        if (x.startsWith("BZh")) {
            return "application/x-bzip2";
        }
        return "unknown >" + x + "<" + buff[0] + " " + buff[1] + " " + buff[2] + " " + buff[3];
    }


// untar from javatar
// bunzip2 from ant-1.6.1

    public void runUrl() throws IOException {
        URL u = new URL(url);
        URLConnection uc = u.openConnection();
        String type = uc.getContentType();
        InputStream is = null;
        try {
            is = uc.getInputStream();
        } catch (IOException e) {
            System.out.println("404 on " + url + " " + e);
            doneURL(url, false);
            return;
        }
        if (type.startsWith("text/html")) {
            if (hopCount > 0) {
                Vector<String> urls = LinkChaser.parseHTML(is,url);
                System.out.println("" + urls.size() + " links in " + url + " " + hopCount);
                for(Iterator<String> i = urls.iterator(); i.hasNext();) {
                    String newURL = i.next();
                    addURL(userID, newURL, false, newURL, hopCount-1);
                }
                doneURL(url, false);
            } else {
                System.out.println("Not Chasing " + url);
                doneURL(url, true);
            }
        } else {
            doneURL(url, false);
            processPlain(userID, originalFile, copyrighted, is);
        }
    }

    public void runFile(File file) throws IOException {
        String fileType = getFileType(file);
        if (fileType.equals("application/pdf")) {
            if (md5 == null) {
                System.out.println("THIS SHOULDN't HAPPEN");
//                md5 = getMD5(file);
            }
            if (hasBeenProcessed(md5)) {
                System.out.println("Found a duplicate (PDF)");
            } else {
                PDFFile.getDir(md5).mkdirs();
                processPDF(file, md5);
                file.renameTo(PDFFile.getFile(md5));
                addToUserFiles(md5);
            }
        } else if (fileType.equals("application/x-gzip")) {
            processPlain(userID, originalFile, copyrighted,
                         new GZIPInputStream(new FileInputStream(file)));
        } else if (fileType.equals("application/zip")) {
            processZip(file);
        } else if (fileType.equals("application/postscript")) {
//            String md5 = getMD5(file);
            if (hasBeenProcessed(md5)) {
                System.out.println("Found a duplicate (PS)");
            } else {
                processPostscript(file);
            }
        } else if (fileType.equals("application/bibtex")) {
            setMessage("File " + originalFile + " unimplemented " + fileType);
        } else if (fileType.equals("application/x-compress")) {
            setMessage("File " + originalFile + " unimplemented " + fileType);
        } else {
            setMessage("File " + originalFile + " unknown mime " + fileType);
        }
    }

    public void runKey() throws IOException {
        Extractor et = null;
        System.out.println("Processing authors of " + key);
        try {
            et = (Extractor) extractionClass.getConstructor().newInstance();
        } catch (Exception e) {
            System.err.println("Could not get a new extractor. We're in poo "+ e);
        }
        et.sortAuthorsOut(key);
    }

    public void run() {
        ClassLoader localClassLoader = null;
        long lastModifiedTime = 0;
        File classFiles = new File("dist/lib/paperBase.jar");
        while (true) {
            getFile();
            try {
                long t = classFiles.lastModified();
                if (t > lastModifiedTime) {
                    lastModifiedTime = t;
                    localClassLoader = null;
                    extractionClass = null;
                    System.gc();
                    System.gc();
                }
                if (localClassLoader == null) {
                    localClassLoader = new ClassReLoader();
                    extractionClass = Class.forName("paper.pdf.ExtractTitle",true,localClassLoader);
                }
                if (type.equals("url")) {
                    runUrl();
                } else if (type.equals("file")) {
                    File file = new File(fileName);
                    runFile(file);
                    file.delete(); // might have been renamed, otherwise delete
                } else if (type.equals("md5")) {
                    processPDF(PDFFile.getFile(md5), md5);
                } else if (type.equals("key")) {
                    runKey();
                } else {
                    System.out.println("Unknown batch job " + type);
                }
            } catch (Exception e) {
                setMessage("File " + originalFile + " not processed - " + e);
                StackTraceElement[] ste = e.getStackTrace();
                for (int i = 0; i < ste.length; i++) {
                    System.err.println(ste[i].toString());
                }
            }
        }
    }

    private void setMessage(String x) {
        System.err.println(x); // someday this shoudl go o the user. Not now. fixme.
    }
}
